/*
 * Copyright 2010-2012 Susanta Tewari. <freecode4susant@users.sourceforge.net>
 *
 * This program is free software: you can redistribute it and/or modify
 * it under the terms of the GNU General Public License as published by
 * the Free Software Foundation, either version 3 of the License, or
 * (at your option) any later version.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with this program.  If not, see <http://www.gnu.org/licenses/>.
 */

package main.genomemap;

import factory.genomemap.FactoryManager;

import genomemap.barratt.BarrattInterferenceModel;
import genomemap.barratt.BarrattModelEstimator;
import genomemap.data.*;
import genomemap.maps.CompressedHybridizationMap;
import genomemap.maps.FullHybridizationMap;
import genomemap.maps.RepDNADistMap;
import genomemap.maps.XMap;
import genomemap.stat.descriptive.DNARepeatsFreqDist;
import genomemap.stat.descriptive.HybridizationStatistics;
import genomemap.stat.tests.RepDNATests;
import genomemap.tagging.Tagger;
import genomemap.tagging.analysis.ComplTagAnalysis;
import genomemap.worker.*;

import gui.genomemap.inputModel.AnnealIModel;
import gui.genomemap.inputModel.ProgCurveIModel;
import gui.genomemap.view.inputView.ProgCurveIView;

import javautil.collections.ArrayUtil;
import javautil.collections.SetUtil;
import javautil.graphics.GraphicsUtil;
import javautil.io.IOUtil;
import javautil.io.LogUtil;
import javautil.lang.MathUtil;
import javautil.lang.StringUtil;

import java.io.File;
import java.io.IOException;
import java.text.DecimalFormat;
import java.util.*;
import java.util.logging.Level;
import java.util.logging.Logger;

/**
 *
 * @author stewari1
 */
public class Main {


    // initial orders: best legacy (probe) orders obtained by applying random cost.
    // Note: the orders were thinned (overlapping probes were removed from the legacy order)
    // the converged probe orders for physical map alone
    // the converged probe orders for joint(integrated) map
    // physical probe spacing

    /** Field description */
    private final double[][] phyProbeSpacings = new double[7][];

    /** Field description */
    private final double[][] jointProbeSpacings = new double[7][];

    /** Field description */
    private boolean probeSpacingsEstimated = false;


    // in (kb : kilo base pairs)

    /** Field description */
    private final int[] phyEstChSize = new int[7];

    /** Field description */
    private final int[] jointEstChSize = new int[7];

    /** Field description */
    private final MainData mainData = new MainDataImpl();

    public static void main(String[] args) {

        Main main = new Main();

        main.createIntegratedGeneMaps();
        main.createIntegratedProbeMaps();
        main.estimateChromosomeSizes();
        main.createRepDNADistMaps();
        main.runProgressCurve();
        main.computeStats();
        main.countOffDiagonals();
        main.runRepDNATests();
        main.runcomplTagAnalysis();
        main.runBatchJobBarratInterferenceModel();
        main.runTagAnalysis();
        main.getRepDNAFreqDist();
        main.getCentromereMapFunPlotData();
        main.computeSeqLHoods();
        main.createFullMaps();
        main.createCompressedMaps();
        main.computeSeqLHoods1();
    }

    public void createFullMaps() {

        for (int i = 0; i < mainData.getJointOrders().length; i++) {

            try {

                System.out.println("Creating Full Map for Chromosome: " + (i + 1));

                File file = new File(mainData.getFullMapDir() + "fullMap_" + (i + 1) + ".txt");
                FullHybridizationMap fullMap = new FullHybridizationMap(i + 1,
                                                   mainData.getJointOrders()[i]);

                IOUtil.writeToFile(file, fullMap.getMapString());

            } catch (IOException ex) {
                Logger.getLogger(Main.class.getName()).log(Level.SEVERE, null, ex);
            }
        }
    }

    public void createCompressedMaps() {

        for (int i = 0; i < mainData.getJointOrders().length; i++) {

            try {

                System.out.println("Creating Compressed Map for Chromosome: " + (i + 1));

                File file = new File(mainData.getCompressedMapDir() + "compressedMap_" + (i + 1)
                                     + ".txt");
                CompressedHybridizationMap compressedMap = new CompressedHybridizationMap(i + 1,
                                                               mainData.getJointOrders()[i]);

                IOUtil.writeToFile(file, compressedMap.getMapString());

            } catch (IOException ex) {
                Logger.getLogger(Main.class.getName()).log(Level.SEVERE, null, ex);
            }
        }
    }

    public void runTagAnalysis() {

        Collection<Gene> allGenes   = new ArrayList<>();
        Collection<Probe> allProbes = new ArrayList<>();

        for (int i = 0; i < 7; i++) {

            int ch_id             = (i + 1);
            int[] jointProbeOrder = mainData.getJointOrders()[i];

            allGenes.addAll(DataSource.getGDataSource(ch_id).getFullGeneSet());
            allProbes.addAll(DataSource.getPDataSource(ch_id, jointProbeOrder).getProbeSet());
        }

        Probe[] probes   = allProbes.toArray(new Probe[0]);
        Gene[] genes     = allGenes.toArray(new Gene[0]);
        Tagger seqTagger = new Tagger(genes, probes);

        seqTagger.addTaggingStrategy(
            FactoryManager.getInstance().getTaggingFactory().getSeqTaggingStrategy());
        System.out.println("Running with sequence tagging strategy");
        seqTagger.tag();
        System.out.println("Total genes tagged: " + seqTagger.getTotalGenesTagged());
        System.out.println("Added: Complementation tagging data");

        Tagger complTagger = new Tagger(genes, probes);

        complTagger.addTaggingStrategy(
            FactoryManager.getInstance().getTaggingFactory().getComplementationTaggStrategy());
        complTagger.tag();
        System.out.println("Total genes tagged: " + complTagger.getTotalGenesTagged());
        System.out.println("Added: Hybridization tagging strategy");

        Tagger hybridTagger = new Tagger(genes, probes);

        hybridTagger.addTaggingStrategy(
            FactoryManager.getInstance().getTaggingFactory().getHybridizationTaginStrategy());
        hybridTagger.tag();
        System.out.println("Total genes tagged: " + hybridTagger.getTotalGenesTagged());
    }

    /**
     * scale the BarrattInterferenceModel from 8k to 159k with 10k increments
     * and report the MSE acroos the range of interference parameter k (0.0 < k < 1.0).
     */
    public void runBatchJobBarratInterferenceModel() {

        BarrattModelEstimator barrattModelEstimator;

        for (double xScale = 8000; xScale < 169000; xScale += 10000) {

            System.out.println("BarrattInterferenceModel for xScale: " + xScale);

            BarrattInterferenceModel barrattModel =
                new BarrattInterferenceModel(FactoryManager.getInstance().getDAOFactory()
                    .getBarrattModelDAO().getBarrattModelData());

            barrattModel.setXScale(xScale);

            SortedMap mseMap = new TreeMap();

            for (int i = 0; i < 11; i++) {

                double k = 0.1 * (i);

                barrattModelEstimator = new BarrattModelEstimator(k);

                barrattModel.setModelEstimator(barrattModelEstimator);

                double mse = barrattModel.getMSE();

                mseMap.put(new Double(mse), new Double(k));
            }

            System.out.println(mseMap);
            System.out.println("");
        }
    }

    public void runcomplTagAnalysis() {

        try {

            GeneCloneTagDAO geneCloneTagDAO =
                FactoryManager.getInstance().getDAOFactory().getGeneCloneTagDAO();
            ComplTagAnalysis complTagAnalysis =
                new ComplTagAnalysis(geneCloneTagDAO.getComplementationData());

            complTagAnalysis.runAnalysis();

        } catch (Exception ex) {
            Logger.getLogger(Main.class.getName()).log(Level.SEVERE, null, ex);
        }
    }

    /**
     * this one computes PLhood merging the probe sequence order
     */
    void computeSeqLHoods1() {

        for (int i = 0; i < 7; i++) {

            int ch_id         = i + 1;
            PDataSource pData = DataSource.getPDataSource(ch_id);
            int[] seqOrder    = DNA_RegionUtil.getProbeSeqOrder(pData.getFullProbeSet());
            int[] mergedOrder = SetUtil.getEmbeddedSeq(mainData.getJointOrders()[i], seqOrder);

            System.out.println("For chromosome: " + ch_id);


//          System.out.println("Best joint order: "); LogUtil.print(jointOrders[i]);
//          System.out.println("Probe Sequence order: "); LogUtil.print(seqOrder);
//          System.out.println("Merged Probe order: ");LogUtil.print(mergedOrder);
            JLikeliHood jHood = new JLikeliHood();

            jHood.setJData(DataSource.getJDataSource(ch_id, mergedOrder));
            jHood.runJHood();
            System.out.println("Joint Likelihood with merged probe order: " + jHood.getLhood());
        }
    }

    /**
     * this one computes GLhood for the gene sequence order
     */
    public void computeSeqLHoods() {

        for (int i = 0; i < 7; i++) {

            int ch_id         = i + 1;
            GDataSource gData = DataSource.getGDataSource(ch_id);
            int[] seqOrder    = DNA_RegionUtil.getGeneSeqOrder(gData.getFullGeneSet());

            System.out.println("Gene Sequence order for chromosome: " + ch_id);
            LogUtil.print(seqOrder);

            GLikeliHood gHood = new GLikeliHood();

            gData.setOrder(seqOrder);
            gData.setRflpImputation(new RFLPImputation(RFLPImputation.POLICY_PROB));
            gHood.setGData(gData);
            gHood.setIterationLimit(30);
            gHood.runGHood();
            System.out.println("Genetic Likelihood with gene Sequence for Chromosome: " + ch_id
                               + "is " + gHood.getLogLikeliHood());
        }
    }

    public void countOffDiagonals() {

        for (int i = 0; i < 1; i++) {

            int ch_id             = i + 1;
            int[] jointProbeOrder = mainData.getJointOrders()[i];
            HybridizationStatistics hybridizationStat = new HybridizationStatistics(ch_id,
                                                            jointProbeOrder);

            System.out.println("Total # of offDiagonals: "
                               + hybridizationStat.getTotalOffDiagonals());
            System.out.println("Total # of signals: "
                               + hybridizationStat.getTotalPositiveSignals());
            System.out.println(
                "The % of offDiagonals: "
                + mainData.getDecimalFormatter().format(
                    hybridizationStat.getOffDiagonalPercentage()));
        }
    }

    public void getCentromereMapFunPlotData() {

        int[] chPoints      = new int[7];
        int[] geneTagCounts = new int[7];

        for (int i = 0; i < 7; i++) {

            int ch_id             = i + 1;
            int[] jointProbeOrder = mainData.getJointOrders()[i];


            // get tagged genes in order
            GDataSource gData = DataSource.getGDataSource(ch_id);
            PDataSource pData = DataSource.getPDataSource(ch_id, jointProbeOrder);

            gData.loadData();
            pData.loadData();


            // with getProbeSet() we tag probes that are in the order else we may tag probes that are not
            // in the order and will be eliminated in jointGeneOrderArray, some genes that were tagged to
            // these outside (out of order) amrkers will remain untagged. As a consequence you may see
            // duplicate Gene s in jointGeneOrderArray. I have changed the code in getGeneOrderArray() which
            // is duplicate proof now. It will never give Gene duplicates but still getProbeSet() method should
            // be used in place of getFullProbeSet().
            LinkedHashSet<Probe> probeSet = pData.getProbeSet();
            LinkedHashSet<Gene> geneSet   = gData.getFullGeneSet();
            Probe[] probes                = probeSet.toArray(new Probe[0]);
            Gene[] genes                  = geneSet.toArray(new Gene[0]);
            Tagger seqTagger              = new Tagger(genes, probes);

            seqTagger.addTaggingStrategy(
                FactoryManager.getInstance().getTaggingFactory().getSeqTaggingStrategy());
            seqTagger.addTaggingStrategy(
                FactoryManager.getInstance().getTaggingFactory().getComplementationTaggStrategy());
            seqTagger.addTaggingStrategy(
                FactoryManager.getInstance().getTaggingFactory().getHybridizationTaginStrategy());
            seqTagger.tag();

            geneTagCounts[i] = seqTagger.getTotalGenesTagged();

            Gene[] jointGeneOrderArray = getTaggedGeneOrderArray(probeSet, geneSet,
                                             jointProbeOrder);

            if (jointGeneOrderArray.length < 3) {

                System.out.println("Too Few tagged markers (count: " + jointGeneOrderArray.length
                                   + ") for Chromosome: " + ch_id);
                System.out.println("Skipping....");

                continue;
            }

            int[] geneOrder = getGeneOrder(jointGeneOrderArray);

            chPoints[i] = geneOrder.length;

            System.out.print("Tagged Gene Orders for Chromosome: " + ch_id + ":  ");
            LogUtil.print(geneOrder);


            // get recombinations by running GLHood
            GLikeliHood gHood = new GLikeliHood();

            gData = DataSource.getGDataSource(ch_id, geneOrder);

            gData.setRflpImputation(new RFLPImputation(RFLPImputation.POLICY_PROB));
            gHood.setGData(gData);
            gHood.setIterationLimit(30);
            gHood.setVerbose(false);
            gHood.runGHood();

            double[] recombs  = gHood.getRecom();
            PLikeliHood pHood = new PLikeliHood();

            pHood.setPData(DataSource.getPDataSource(ch_id, jointProbeOrder));
            pHood.runPHood();

            double[] probeSpacings      = pHood.getProbeSpacing();
            double[] phyDistances       = new double[recombs.length];
            DecimalFormat decimalFormat = new DecimalFormat("0.#####");

            System.out.println("\nCentromere Effect Plot Data for Chromosome: " + ch_id);

            for (int j = 0; j < jointGeneOrderArray.length - 1; j++) {

                Gene gene1      = jointGeneOrderArray[j];
                Gene gene2      = jointGeneOrderArray[j + 1];
                int probeOrder1 = gene1.getTaggedCloneSet().iterator().next().getOrder();
                int probeOrder2 = gene2.getTaggedCloneSet().iterator().next().getOrder();

                phyDistances[j] = getPhysicalDistance(jointProbeOrder, probeSpacings, probeOrder1,
                        probeOrder2, 34);

                System.out.println(gene1.getLabel());
                System.out.println("    " + "r: " + decimalFormat.format(recombs[j]) + "  " + "p: "
                                   + decimalFormat.format(phyDistances[j]));
                System.out.println(gene2.getLabel());
            }


            // create R by P matrix for mapping function plotting
            String[][] rpMat = new String[recombs.length][recombs.length];

            StringUtil.setWithEmptyStr(rpMat, "");

            for (int j = 0; j < recombs.length; j++) {

                for (int k = j; k < recombs.length; k++) {

                    if (k == j) {

                        rpMat[j][k] = "(" + decimalFormat.format(recombs[j]) + ","
                                      + decimalFormat.format(phyDistances[j]) + ")";
                    }

                    if (k != j) {

                        rpMat[j][k] = "(" + decimalFormat.format(recombs[k] + recombs[j]) + ","
                                      + decimalFormat.format(phyDistances[k] + phyDistances[j])
                                      + ")";
                    }
                }
            }

            System.out.println("\nR by P matrix for mapping function plot\n");

            for (int j = 0; j < jointGeneOrderArray.length - 1; j++) {

                Gene gene1 = jointGeneOrderArray[j];
                Gene gene2 = jointGeneOrderArray[j + 1];

                System.out.print("[" + gene1.getLabel() + "," + gene2.getLabel() + "]" + "  ");
            }

            System.out.println("");
            LogUtil.print(rpMat, 2, 1, null);
        }

        System.out.println("r-p points chromosome wise: ");
        LogUtil.print(chPoints);
        System.out.println("gene tag counts chromosome wise: ");
        LogUtil.print(geneTagCounts);
        System.out.println("Total r-p points: " + MathUtil.Sum(chPoints));
        System.out.println("Total gene tag points: " + MathUtil.Sum(geneTagCounts));
    }

    /**
     * gets the physical distance between probe with order probeOrder1 and probeOrder2.
     *
     *
     * @param probeOrder description
     * @param probeSpacings description
     * @param probeOrder1 description
     * @param probeOrder2 description
     * @param stdProbeSize description
     *
     * @return description
     */
    private double getPhysicalDistance(int[] probeOrder, double[] probeSpacings, int probeOrder1,
                                       int probeOrder2, int stdProbeSize) {

        double distance     = 0.0;
        boolean probe1Found = false;
        boolean probe2Found = false;

        if (probeOrder1 == probeOrder2) {

            System.out.println(
                "from getPhysicalDistance() in Main: probeOrder1 and probeOrder2 ae same");
        }

        for (int i = 0; i < probeOrder.length; i++) {

            if (!probe1Found && (probeOrder[i] == probeOrder1)) {
                probe1Found = true;
            }

            if (probe1Found) {
                distance += probeSpacings[i + 1] + stdProbeSize;
            }

            if (probe1Found &&!probe2Found && (probeOrder[i] == probeOrder2)) {
                probe2Found = true;
            }

            if (probe2Found) {
                break;
            }
        }

        return distance;
    }

    /**
     * Computing various sttaistics needed in the paper
     */
    public void computeStats() {

        if (!probeSpacingsEstimated) {
            estimateProbeSpacings();
        }


        // compute total number of probes and their total +ve hybridizations
        // and computing the resolution of the joint map
        int totalProbecount     = 0;
        int totalPosHybrid      = 0;
        double totalSpacingSize = 0;

        for (int i = 0; i < 7; i++) {

            int ch_id             = i + 1;
            int[] jointProbeOrder = mainData.getJointOrders()[i];

            totalProbecount += jointProbeOrder.length;

            int[][] hybridData = DataSource.getPDataSource(ch_id).getHybridData();

            for (int j = 0; j < jointProbeOrder.length; j++) {

                for (int k = 0; k < hybridData.length; k++) {

                    if (hybridData[k][jointProbeOrder[j] - 1] == 1) {
                        totalPosHybrid++;
                    }
                }
            }

            totalSpacingSize += MathUtil.Sum(jointProbeSpacings[i])
                                - (jointProbeSpacings[i][0]
                                   + jointProbeSpacings[i][jointProbeSpacings[i].length - 1]);
        }

        System.out.println("Total probe count: " + totalProbecount);
        System.out.println("Total positive hybridizations : " + totalPosHybrid);
        System.out.println("Average positive hybridization is : "
                           + (int) (double) totalPosHybrid / totalProbecount);
        System.out.println("Joint Map resolution: "
                           + (34 * totalProbecount + totalSpacingSize) / (2 * totalProbecount));
    }

    public void runRepDNATests() {


        // run test without ch-V first
        int[] ch_ids        = new int[6];
        int[][] probeOrders = new int[6][];

        for (int i = 0; i < 6; i++) {

            if (i != 4) {


                // exclude ch-V
                ch_ids[i]      = (i + 1);
                probeOrders[i] = mainData.getJointOrders()[i];
            }

            if (i >= 4) {

                ch_ids[i]      = (i + 2);
                probeOrders[i] = mainData.getJointOrders()[i + 1];
            }
        }

        System.out.println("Running Repetitive DNA Independence test for all but ch: 5");

        RepDNATests repDNAIndTest = new RepDNATests(ch_ids, probeOrders);

        repDNAIndTest.runIndependenceTest();
        System.out.println("Running Repetitive DNA Homogeneity test for all but ch: 5");
        repDNAIndTest.runHomogeneityTest();


        // run test for all:
        ch_ids      = new int[7];
        probeOrders = new int[7][];

        for (int i = 0; i < 7; i++) {

            ch_ids[i]      = (i + 1);
            probeOrders[i] = mainData.getJointOrders()[i];
        }

        System.out.println("Running Repetitive DNA independence test for all  chromosomes");

        repDNAIndTest = new RepDNATests(ch_ids, probeOrders);

        repDNAIndTest.runIndependenceTest();
        System.out.println("Running Repetitive DNA Homogeneity test for all  chromosomes");
        repDNAIndTest.runHomogeneityTest();
    }

    public void getRepDNAFreqDist() {

        int[] ch_ids        = new int[7];
        int[][] probeOrders = new int[7][];

        for (int i = 0; i < 7; i++) {

            ch_ids[i]      = (i + 1);
            probeOrders[i] = mainData.getJointOrders()[i];
        }

        DNARepeatsFreqDist repeatsFreqDist = new DNARepeatsFreqDist(ch_ids, probeOrders);

        repeatsFreqDist.setDistType(DNARepeatsFreqDist.DIST_TYPE_SRA_TRANSITION);
        System.out.println(repeatsFreqDist.getFreqTableString());
        System.out.println(repeatsFreqDist.getFreqPercentTableString());
    }

    /**
     * The map format: (Genes only)
     * GeneticOrder---InegeratedProbeOrder---PhysicalOrder
     * Note that in InegeratedProbeOrder and PhysicalOrder we have to get the tagged
     * genes in the probe order.
     */
    public void createIntegratedGeneMaps() {

        System.out.println("creating Integrated Gene Maps");

        for (int i = 0; i < 7; i++) {

            try {

                int ch_id = i + 1;


//              System.out.println("Working on Chromosome - " + ch_id);
                int[] geneOrder          = mainData.getGeneOrders()[i];
                int[] jointProbeOrder    = mainData.getJointOrders()[i];
                int[] physicalProbeOrder = mainData.getPhyOrders()[i];
                GDataSource gData        = DataSource.getGDataSource(ch_id);
                PDataSource pData        = DataSource.getPDataSource(ch_id);

                gData.loadData();
                pData.loadData();

                LinkedHashSet<Probe> probeSet   = pData.getFullProbeSet();
                LinkedHashSet<Gene> geneSet     = new LinkedHashSet<>();
                LinkedHashSet<Gene> fullGeneSet = gData.getFullGeneSet();

                for (Gene gene : fullGeneSet) {

                    if (gene.hasSeqInfo()) {
                        geneSet.add(gene);
                    }
                }


                // configure tagging among probes and genes
                Iterator<Probe> itrProbe = probeSet.iterator();

                while (itrProbe.hasNext()) {

                    Probe probe = itrProbe.next();

                    if (probe.hasSeqInfo()) {

                        Iterator<Gene> itrGene = geneSet.iterator();

                        while (itrGene.hasNext()) {

                            Gene gene = itrGene.next();

                            if (!gene.isTagged() && probe.isOverLapping(gene)) {

                                gene.addTaggedClone(probe);
                                probe.addTaggedGene(gene);

                                break;
                            }
                        }
                    }
                }

                int[] jointGeneOrder = getGeneOrder(probeSet, geneSet, jointProbeOrder);
                int[] phyGeneOrder = getGeneOrder(probeSet, geneSet, physicalProbeOrder);
                String[] mapHeaders = { "Using GLHood", "Using JLHood", "Using PLHood" };
                boolean[] anyCross                     = { true, true };
                LinkedHashSet<String>[] labelSetsGiven = new LinkedHashSet[3];

                labelSetsGiven[0] = DNA_RegionUtil.getGeneLabels(fullGeneSet, geneOrder);
                labelSetsGiven[1] = DNA_RegionUtil.getGeneLabels(fullGeneSet, jointGeneOrder);
                labelSetsGiven[2] = DNA_RegionUtil.getGeneLabels(fullGeneSet, phyGeneOrder);

                XMap xpMap = new XMap();

                xpMap.instantiate(labelSetsGiven, mapHeaders, anyCross);

                File file = new File(mainData.getIntegratedGeneMapDir() + "IGeneMap_" + ch_id
                                     + ".jpg");

                GraphicsUtil.saveImage(file, GraphicsUtil.IMAGE_FORMAT_JPEG, xpMap,
                                       xpMap.getWidth(), xpMap.getHeight());
                System.out.println(
                    "Crossing Info for Ch: " + ch_id + "  "
                    + IOUtil.toString(ArrayUtil.getIntegerArray(xpMap.getCrossCounts()), ","));

            } catch (Exception ex) {
                Logger.getLogger(Main.class.getName()).log(Level.SEVERE, null, ex);
            }
        }
    }

    /**
     * The map format: (Probes only)
     * GeneticOrder---InegeratedProbeOrder---PhysicalOrder
     * Note that in GeneticOrder we have to get the tagged
     * probes in the gene order.
     */
    public void createIntegratedProbeMaps() {

        System.out.println("creating Integrated Probe Maps");

        for (int i = 0; i < 7; i++) {

            try {

                int ch_id = i + 1;


//              System.out.println("Working on Chromosome - " + ch_id);
                int[] geneOrder       = mainData.getGeneOrders()[i];
                int[] jointProbeOrder = mainData.getJointOrders()[i];
                GDataSource gData     = DataSource.getGDataSource(ch_id);
                PDataSource pData     = DataSource.getPDataSource(ch_id);

                gData.loadData();
                pData.loadData();

                LinkedHashSet<Probe> fullProbeSet = pData.getFullProbeSet();
                LinkedHashSet<Gene> geneSet       = new LinkedHashSet<>();
                LinkedHashSet<Gene> fullGeneSet   = gData.getFullGeneSet();

                for (Gene gene : fullGeneSet) {

                    if (gene.hasSeqInfo()) {
                        geneSet.add(gene);
                    }
                }


                // configure tagging among probes and genes
                Iterator<Probe> itrProbe = fullProbeSet.iterator();

                while (itrProbe.hasNext()) {

                    Probe probe = itrProbe.next();

                    if (probe.hasSeqInfo()) {

                        Iterator<Gene> itrGene = geneSet.iterator();

                        while (itrGene.hasNext()) {

                            Gene gene = itrGene.next();

                            if (!gene.isTagged() && probe.isOverLapping(gene)) {

                                gene.addTaggedClone(probe);
                                probe.addTaggedGene(gene);

                                break;
                            }
                        }
                    }
                }

                int[] geneBasedProbeOrder = getProbeOrder(fullProbeSet, geneSet, geneOrder);
                int[] seqProbeOrder = DNA_RegionUtil.getProbeSeqOrder(fullProbeSet);
                String[] mapHeaders = { "Using GLHood", "Using JLHood", "Using Sequence" };
                boolean[] anyCross                     = { true, true };
                LinkedHashSet<String>[] labelSetsGiven = new LinkedHashSet[3];

                labelSetsGiven[0] = DNA_RegionUtil.getProbeLabels(fullProbeSet,
                        geneBasedProbeOrder);
                labelSetsGiven[1] = DNA_RegionUtil.getProbeLabels(fullProbeSet, jointProbeOrder);
                labelSetsGiven[2] = DNA_RegionUtil.getProbeLabels(fullProbeSet, seqProbeOrder);

                XMap xpMap = new XMap();

                xpMap.instantiate(labelSetsGiven, mapHeaders, anyCross);

                File file = new File(mainData.getIntegratedProbeMapDir() + "IProbeMap_" + ch_id
                                     + ".jpg");

                GraphicsUtil.saveImage(file, GraphicsUtil.IMAGE_FORMAT_JPEG, xpMap,
                                       xpMap.getWidth(), xpMap.getHeight());
                System.out.println(
                    "Crossing Info for Ch: " + ch_id + "  "
                    + IOUtil.toString(ArrayUtil.getIntegerArray(xpMap.getCrossCounts()), ","));

            } catch (Exception ex) {
                Logger.getLogger(Main.class.getName()).log(Level.SEVERE, null, ex);
            }
        }
    }

    /**
     * Length of Spacing array = # of probes + 2(accouting for the first  and last space)
     * Spacings must have been estimated before this is run.
     */
    public void estimateChromosomeSizes() {

        if (!probeSpacingsEstimated) {
            estimateProbeSpacings();
        }

        System.out.println("Estimating Chromosome Sizes using Joint LHood");

        for (int i = 0; i < 7; i++) {

            double estSize = 34 * (jointProbeSpacings[i].length - 1)
                             + MathUtil.Sum(jointProbeSpacings[i])
                             - (jointProbeSpacings[i][0]
                                + jointProbeSpacings[i][jointProbeSpacings[i].length - 1]);

            jointEstChSize[i] = (int) estSize;

            System.out.println(
                "Chromosome-" + (i + 1) + " Estimate Size: "
                + mainData.getDecimalFormatter().format(new Double(estSize)) + " Actual Size: "
                + mainData.getDecimalFormatter().format(new Double(mainData.getActualChSize()[i])));
        }

        System.out.println("Estimating Chromosome Sizes using Physical LHood");

        for (int i = 0; i < 7; i++) {

            double estSize = 34 * (phyProbeSpacings[i].length - 1)
                             + MathUtil.Sum(phyProbeSpacings[i])
                             - (phyProbeSpacings[i][0]
                                + phyProbeSpacings[i][phyProbeSpacings[i].length - 1]);

            phyEstChSize[i] = (int) estSize;

            System.out.println(
                "Chromosome-" + (i + 1) + " Estimate Size: "
                + mainData.getDecimalFormatter().format(new Double(estSize)) + " Actual Size: "
                + mainData.getDecimalFormatter().format(new Double(mainData.getActualChSize()[i])));
        }
    }

    public void createRepDNADistMaps() {

        if (!probeSpacingsEstimated) {
            estimateProbeSpacings();
        }

        System.out.println("Creating Repeated DNA Distribution Map for Physical Probe orders");

        RepDNADistMap repDNADistMap = new RepDNADistMap();

        repDNADistMap.instantiate("Repeated DNA Distribution Map (Physical)", 34,
                                  mainData.getPhyOrders(),
                                  ArrayUtil.convertDoubleToIntegerArray(phyProbeSpacings));

        File file = new File(mainData.getResultsDir() + "RepDNADistMap_Physical" + ".jpg");

        GraphicsUtil.saveImage(file, GraphicsUtil.IMAGE_FORMAT_JPEG, repDNADistMap,
                               repDNADistMap.getWidth(), repDNADistMap.getHeight());
        System.out.println("Creating Repeated DNA Distribution Map for Joint Probe orders");

        repDNADistMap = new RepDNADistMap();

        repDNADistMap.instantiate("Repeated DNA Distribution Map (Joint)", 34,
                                  mainData.getJointOrders(),
                                  ArrayUtil.convertDoubleToIntegerArray(jointProbeSpacings));

        file = new File(mainData.getResultsDir() + "RepDNADistMap_Joint" + ".jpg");

        GraphicsUtil.saveImage(file, GraphicsUtil.IMAGE_FORMAT_JPEG, repDNADistMap,
                               repDNADistMap.getWidth(), repDNADistMap.getHeight());
    }

    public void runProgressCurve() {


//      System.out.println("Running Progress Curve using Physical Likelihood");
//      for (int chIndex = 0; chIndex < 7; chIndex++) {
//            System.out.println("Chromosome: " + (chIndex + 1));
//            runProgressCurve(initOrders[chIndex],chIndex+1,Annealable.ANNEAL_CRITERION_PHOOD);
//      }
//      System.out.println("Running Progress Curve using Joint Likelihood");
//      for (int chIndex = 0; chIndex < 7; chIndex++) {
//          
//            System.out.println("Chromosome: " + (chIndex + 1));
//            runProgressCurve(initOrders[chIndex],chIndex+1,Annealable.ANNEAL_CRITERION_JHOOD);
//          
//      }
        System.out.println(
            "Progress Curve using Joint Likelihood,Initiaized with RC for orginal thinned probe order");

        int[] chSizes          = {
            237, 210, 181, 226, 285, 176, 165
        };
        int[][] orgProbeOrders = new int[7][];

        for (int i = 0; i < orgProbeOrders.length; i++) {

            orgProbeOrders[i] = ArrayUtil.IntegerSequence(chSizes[i], false);
        }

        int[][] thinnedProbeOrders = createThinProbeOrders(orgProbeOrders);

        for (int chIndex = 0; chIndex < 7; chIndex++) {

            System.out.println("Chromosome: " + (chIndex + 1));
            runProgressCurve(thinnedProbeOrders[chIndex], chIndex + 1,
                             Annealable.ANNEAL_CRITERION_JHOOD);
        }
    }

    /**
     * Method description
     *
     * @param order description
     * @param ch_id description
     * @param annealCriterion description
     */
    private void runProgressCurve(int[] order, int ch_id, int annealCriterion) {

        ProgCurveIModel inputModel = new ProgCurveIModel();

        inputModel.setLinkageGroup(ch_id);
        inputModel.setSegProbeCount(10);
        inputModel.setOrder(order);
        inputModel.setSegmentCount(order.length / inputModel.getSegProbeCount());
        inputModel.setAnnealCriterion(annealCriterion);
        inputModel.setAnnealMachineType(AnnealMachine.MACHINE_TYPE_SIMULATED_ANNEALING);

        AnnealIModel annealIModel = new AnnealIModel();

        inputModel.setAnnealInputOptions(annealIModel);


        // set the anneal options
        annealIModel.setGIterationLimit(1);
        annealIModel.setGConvgEpsilon(0.0001);

        RFLPImputation rflpImputation = new RFLPImputation();

        rflpImputation.setPolicyIndex(RFLPImputation.POLICY_PROB);
        rflpImputation.setImputationProb(0.5);
        annealIModel.setRflpImputation(rflpImputation);
        annealIModel.setMASTER_ITER_COUNT(500000);
        annealIModel.setTEMP_FRICTION(0.95);
        annealIModel.setTEMP_ITER_COUNT(10000);
        annealIModel.setSTART_TEMP(50.0);
        annealIModel.setMAX_UNBEATEN_COUNT(2000);
        annealIModel.setMAX_BEATEN_COUNT(500);
        annealIModel.setPercent(0.001);
        annealIModel.setLinkageGroup(inputModel.getLinkageGroup());

        Integer[] entireProbeOrder = ArrayUtil.getIntegerArray(inputModel.getOrder());
        int segmentLen             = inputModel.getSegProbeCount();
        int segments               = inputModel.getSegmentCount();

        try {

            for (int segment = 1; segment <= segments - 1; segment++) {

                int endPos                       = segment * segmentLen;
                int[] segmentOrder =
                    ArrayUtil.getIntegerArray(ArrayUtil.getSubArray(entireProbeOrder,
                        Integer.class, 1, endPos));
                AnnealIModel segmentAnnealIModel = (AnnealIModel) annealIModel.clone();

                segmentAnnealIModel.setOrder(segmentOrder);
                inputModel.addAnnealIModel(segmentAnnealIModel);
            }


            // the last segment
            int endPos                       = entireProbeOrder.length;
            int[] segmentOrder = ArrayUtil.getIntegerArray(ArrayUtil.getSubArray(entireProbeOrder,
                                     Integer.class, 1, endPos));
            AnnealIModel segmentAnnealIModel = (AnnealIModel) annealIModel.clone();

            segmentAnnealIModel.setOrder(segmentOrder);
            inputModel.addAnnealIModel(segmentAnnealIModel);

        } catch (CloneNotSupportedException ex) {
            Logger.getLogger(ProgCurveIView.class.getName()).log(Level.SEVERE, null, ex);
        }

        ProgCurve progCurve = new ProgCurve(inputModel);

        try {
            progCurve.run();
        } catch (Exception exception) {
            exception.printStackTrace();
        }
    }

    /**
     * Method description
     */
    private void estimateProbeSpacings() {

        System.out.println("Estimating probe spacings for the physical order");
        estimateProbeSpacings(mainData.getPhyOrders(), phyProbeSpacings);
        System.out.println("Estimating probe spacings for the joint physical order");
        estimateProbeSpacings(mainData.getJointOrders(), jointProbeSpacings);

        probeSpacingsEstimated = true;
    }

    /**
     * Method description
     *
     * @param probeOrders description
     * @param probeSpacings description
     */
    private void estimateProbeSpacings(int[][] probeOrders, double[][] probeSpacings) {

        for (int i = 0; i < 7; i++) {

            PLikeliHood pHood = new PLikeliHood();

            pHood.setPData(DataSource.getPDataSource((i + 1), probeOrders[i]));
            pHood.runPHood();

            probeSpacings[i] = pHood.getProbeSpacing();
        }
    }

    /**
     * Method description
     *
     * @param chProbeOrders description
     *
     * @return description
     */
    private int[][] createThinProbeOrders(int[][] chProbeOrders) {

        int[][] thinOrders = new int[7][];

        for (int i = 0; i < 7; i++) {

            int[] probeOrder        = chProbeOrders[i];
            PDataSource pData       = DataSource.getPDataSource(i + 1);
            int[][] hybridData      = pData.getHybridData();
            Set<Probe> probeSet     = pData.getFullProbeSet();
            Set<Clone> cloneSet     = pData.getCloneSet();
            List<Integer> orderList = new ArrayList<>();


            // remove probes that hybridize to a given probe
            String orderStr = "";

            for (int index = 0; index < probeOrder.length; index++) {

                Probe probe = getProbe(probeSet, probeOrder[index]);

                if (probe == null) {


                    // this means it was previously released..
                    continue;
                }

                int cloneIndex = getCloneindex(cloneSet, probe.getLabel());

                if (cloneIndex == -1) {


                    // System.out.println("Probe: " + probe.getLabel() + " isn`t found in clones");
                } else {

                    for (int probeIndex = 0; probeIndex < hybridData[0].length; probeIndex++) {

                        if ((probeIndex != probe.getOrder())
                                && (hybridData[cloneIndex][probeIndex] == 1)) {
                            probeSet.remove(getProbe(probeSet, probeIndex + 1));
                        }
                    }
                }

                if (index != probeOrder.length - 1) {
                    orderStr += probe.getOrder() + ",";
                } else {
                    orderStr += probe.getOrder() + "\n";
                }

                orderList.add(new Integer(probe.getOrder()));
            }

            thinOrders[i] = ArrayUtil.getIntegerArray(orderList.toArray(new Integer[0]));

            System.out.println("Thin Probe Order: Ch: " + (i + 1) + "   " + orderStr);
            System.out.println("Chromosome: " + (i + 1) + " probe count: " + probeOrder.length
                               + " new probe count is : " + orderList.size());
        }

        return thinOrders;
    }

    /**
     * Method description
     *
     * @param cloneSet description
     * @param label description
     *
     * @return description
     */
    private int getCloneindex(Set<Clone> cloneSet, String label) {

        Iterator<Clone> itr = cloneSet.iterator();

        while (itr.hasNext()) {

            Clone clone = itr.next();

            if (clone.getLabel().equals(label)) {
                return clone.getOrder();
            }
        }

        return -1;
    }

    /**
     * Method description
     *
     * @param probeSet description
     * @param probeIndex description
     *
     * @return description
     */
    private Probe getProbe(Set<Probe> probeSet, int probeIndex) {

        Iterator<Probe> itr = probeSet.iterator();

        while (itr.hasNext()) {

            Probe probe = itr.next();

            if (probe.getOrder() == probeIndex) {
                return probe;
            }
        }

        return null;
    }

    /**
     * Method description
     *
     * @param probeSet description
     * @param geneSet description
     * @param probeOrder description
     *
     * @return description
     */
    private Gene[] getTaggedGeneOrderArray(LinkedHashSet<Probe> probeSet,
            LinkedHashSet<Gene> geneSet, int[] probeOrder) {

        Gene[] geneArray = geneSet.toArray(new Gene[0]);
        boolean[] placed = new boolean[geneArray.length];    // default is false

        for (int geneIndex = 0; geneIndex < geneArray.length; geneIndex++) {


            // for genes that are not tagged  will remain in place and hence placed wiill be true for them
            if (!geneArray[geneIndex].isTagged()) {
                placed[geneIndex] = true;
            }
        }


        // order the probes into probeArray
        List<Probe> reqProbeList = new ArrayList<>();

        for (int probeIndex1 = 0; probeIndex1 < probeOrder.length; probeIndex1++) {

            for (Iterator<Probe> itr = probeSet.iterator(); itr.hasNext(); ) {

                Probe probe = itr.next();

                if (probeOrder[probeIndex1] == probe.getOrder()) {

                    reqProbeList.add(probe);

                    break;
                }
            }
        }


        // order the genes into geneArray
        for (int probeIndex = 0; probeIndex < reqProbeList.size(); probeIndex++) {

            Probe probe = reqProbeList.get(probeIndex);

            if (probe.isTagged()) {

                for (int geneIndex = 0; geneIndex < geneArray.length; geneIndex++) {

                    if (!placed[geneIndex] && geneArray[geneIndex].isTagged()) {

                        Gene gene = probe.getTaggedGeneSet().iterator().next();

                        placed[geneIndex]    = true;
                        geneArray[geneIndex] = gene;

                        break;
                    }
                }
            }
        }


        // a check: if there is any false in placed, that signals an error
        // all genes that were tagged tagged must have been claimed by their
        // corresponding probes their probe order. failing to do so indicates data integrity loss
        for (int i = 0; i < placed.length; i++) {

            if (!placed[i]) {

                System.out.println("From jbcb_3.getTaggedGeneOrderArray(): ");
                System.out.println(":Some tagged genes not claimed by their probes..");
                System.out.println("possible data integrity loss..");
                System.exit(0);

            } else {}
        }

        LinkedHashSet<Gene> taggedGeneSet = new LinkedHashSet<>();

        for (Gene gene : geneArray) {

            if (gene.isTagged()) {
                taggedGeneSet.add(gene);
            }
        }


        // if you have even # of genes, drop the first one for lhood calculatio to make it odd.
        if (taggedGeneSet.size() % 2 == 0) {
            taggedGeneSet.remove(taggedGeneSet.iterator().next());
        }

        return taggedGeneSet.toArray(new Gene[0]);
    }

    /**
     * Method description
     *
     * @param probeSet description
     * @param geneSet description
     * @param probeOrder description
     *
     * @return description
     */
    private int[] getGeneOrder(LinkedHashSet<Probe> probeSet, LinkedHashSet<Gene> geneSet,
                               int[] probeOrder) {
        return getGeneOrder(getTaggedGeneOrderArray(probeSet, geneSet, probeOrder));
    }

    /**
     * Method description
     *
     * @param geneArray description
     *
     * @return description
     */
    private int[] getGeneOrder(Gene[] geneArray) {

        int[] geneOrder = new int[geneArray.length];

        for (int geneIndex = 0; geneIndex < geneArray.length; geneIndex++) {

            geneOrder[geneIndex] = geneArray[geneIndex].getOrder();
        }

        return geneOrder;
    }

    /**
     * Method description
     *
     * @param probeSet description
     * @param geneSet description
     * @param geneOrder description
     *
     * @return description
     */
    private int[] getProbeOrder(LinkedHashSet<Probe> probeSet, LinkedHashSet<Gene> geneSet,
                                int[] geneOrder) {

        Probe[] probeArray = probeSet.toArray(new Probe[0]);
        boolean[] placed   = new boolean[probeArray.length];    // default is false


        //
        List<Gene> orderedGeneList = new ArrayList<>();

        for (int geneIndex1 = 0; geneIndex1 < geneOrder.length; geneIndex1++) {

            for (Iterator<Gene> itr = geneSet.iterator(); itr.hasNext(); ) {

                Gene gene = itr.next();

                if (geneOrder[geneIndex1] == gene.getOrder()) {
                    orderedGeneList.add(gene);
                }
            }
        }


        // order the probes
        for (int geneIndex = 0; geneIndex < orderedGeneList.size(); geneIndex++) {

            Gene gene = orderedGeneList.get(geneIndex);

            if (gene.isTagged()) {

                for (int probeIndex = 0; probeIndex < probeArray.length; probeIndex++) {

                    if (!placed[probeIndex] && probeArray[probeIndex].isTagged()) {

                        Probe probe = (Probe) gene.getTaggedCloneSet().iterator().next();

                        placed[probeIndex]     = true;
                        probeArray[probeIndex] = probe;

                        break;
                    }
                }
            }
        }

        int[] probeOrder = new int[probeArray.length];

        for (int probeIndex = 0; probeIndex < probeArray.length; probeIndex++) {

            probeOrder[probeIndex] = probeArray[probeIndex].getOrder();
        }

        return probeOrder;
    }
}
